{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import torch\n",
    "import ast"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load FMA Metadata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "metadata_folder = '../data/fma_metadata'\n",
    "genres = pd.read_csv(os.path.join(metadata_folder, 'genres.csv'), index_col=0)\n",
    "tracks = pd.read_csv(os.path.join(metadata_folder, 'tracks.csv'), index_col=0, header=[0, 1])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Genres"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#tracks</th>\n",
       "      <th>parent</th>\n",
       "      <th>title</th>\n",
       "      <th>top_level</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>genre_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8693</td>\n",
       "      <td>38</td>\n",
       "      <td>Avant-Garde</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5271</td>\n",
       "      <td>0</td>\n",
       "      <td>International</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1752</td>\n",
       "      <td>0</td>\n",
       "      <td>Blues</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4126</td>\n",
       "      <td>0</td>\n",
       "      <td>Jazz</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4106</td>\n",
       "      <td>0</td>\n",
       "      <td>Classical</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          #tracks  parent          title  top_level\n",
       "genre_id                                           \n",
       "1            8693      38    Avant-Garde         38\n",
       "2            5271       0  International          2\n",
       "3            1752       0          Blues          3\n",
       "4            4126       0           Jazz          4\n",
       "5            4106       0      Classical          5"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genres.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16 top-level genres\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#tracks</th>\n",
       "      <th>parent</th>\n",
       "      <th>title</th>\n",
       "      <th>top_level</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>genre_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>38154</td>\n",
       "      <td>0</td>\n",
       "      <td>Experimental</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>34413</td>\n",
       "      <td>0</td>\n",
       "      <td>Electronic</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>32923</td>\n",
       "      <td>0</td>\n",
       "      <td>Rock</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1235</th>\n",
       "      <td>14938</td>\n",
       "      <td>0</td>\n",
       "      <td>Instrumental</td>\n",
       "      <td>1235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>13845</td>\n",
       "      <td>0</td>\n",
       "      <td>Pop</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>12706</td>\n",
       "      <td>0</td>\n",
       "      <td>Folk</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>8389</td>\n",
       "      <td>0</td>\n",
       "      <td>Hip-Hop</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5271</td>\n",
       "      <td>0</td>\n",
       "      <td>International</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4126</td>\n",
       "      <td>0</td>\n",
       "      <td>Jazz</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>4106</td>\n",
       "      <td>0</td>\n",
       "      <td>Classical</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1987</td>\n",
       "      <td>0</td>\n",
       "      <td>Country</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>1876</td>\n",
       "      <td>0</td>\n",
       "      <td>Spoken</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1752</td>\n",
       "      <td>0</td>\n",
       "      <td>Blues</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>1499</td>\n",
       "      <td>0</td>\n",
       "      <td>Soul-RnB</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>868</td>\n",
       "      <td>0</td>\n",
       "      <td>Old-Time / Historic</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>730</td>\n",
       "      <td>0</td>\n",
       "      <td>Easy Listening</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          #tracks  parent                title  top_level\n",
       "genre_id                                                 \n",
       "38          38154       0         Experimental         38\n",
       "15          34413       0           Electronic         15\n",
       "12          32923       0                 Rock         12\n",
       "1235        14938       0         Instrumental       1235\n",
       "10          13845       0                  Pop         10\n",
       "17          12706       0                 Folk         17\n",
       "21           8389       0              Hip-Hop         21\n",
       "2            5271       0        International          2\n",
       "4            4126       0                 Jazz          4\n",
       "5            4106       0            Classical          5\n",
       "9            1987       0              Country          9\n",
       "20           1876       0               Spoken         20\n",
       "3            1752       0                Blues          3\n",
       "14           1499       0             Soul-RnB         14\n",
       "8             868       0  Old-Time / Historic          8\n",
       "13            730       0       Easy Listening         13"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('{} top-level genres'.format(len(genres['top_level'].unique())))\n",
    "genres.loc[genres['top_level'].unique()].sort_values('#tracks', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14 subgenres of \"experimental\"\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>#tracks</th>\n",
       "      <th>parent</th>\n",
       "      <th>title</th>\n",
       "      <th>top_level</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>genre_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>8693</td>\n",
       "      <td>38</td>\n",
       "      <td>Avant-Garde</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>914</td>\n",
       "      <td>38</td>\n",
       "      <td>Novelty</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>774</td>\n",
       "      <td>38</td>\n",
       "      <td>Audio Collage</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>3237</td>\n",
       "      <td>38</td>\n",
       "      <td>Field Recordings</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>7268</td>\n",
       "      <td>38</td>\n",
       "      <td>Noise</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>6110</td>\n",
       "      <td>38</td>\n",
       "      <td>Electroacoustic</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>2546</td>\n",
       "      <td>38</td>\n",
       "      <td>Drone</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>1511</td>\n",
       "      <td>38</td>\n",
       "      <td>Unclassifiable</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>186</th>\n",
       "      <td>682</td>\n",
       "      <td>38</td>\n",
       "      <td>Sound Poetry</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>224</th>\n",
       "      <td>1916</td>\n",
       "      <td>38</td>\n",
       "      <td>Sound Collage</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>247</th>\n",
       "      <td>2957</td>\n",
       "      <td>38</td>\n",
       "      <td>Musique Concrete</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>250</th>\n",
       "      <td>4261</td>\n",
       "      <td>38</td>\n",
       "      <td>Improv</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>456</th>\n",
       "      <td>1392</td>\n",
       "      <td>38</td>\n",
       "      <td>Minimalism</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>514</th>\n",
       "      <td>1414</td>\n",
       "      <td>38</td>\n",
       "      <td>Sound Art</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          #tracks  parent             title  top_level\n",
       "genre_id                                              \n",
       "1            8693      38       Avant-Garde         38\n",
       "6             914      38           Novelty         38\n",
       "22            774      38     Audio Collage         38\n",
       "30           3237      38  Field Recordings         38\n",
       "32           7268      38             Noise         38\n",
       "41           6110      38   Electroacoustic         38\n",
       "47           2546      38             Drone         38\n",
       "125          1511      38    Unclassifiable         38\n",
       "186           682      38      Sound Poetry         38\n",
       "224          1916      38     Sound Collage         38\n",
       "247          2957      38  Musique Concrete         38\n",
       "250          4261      38            Improv         38\n",
       "456          1392      38        Minimalism         38\n",
       "514          1414      38         Sound Art         38"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "experimental = genres[genres['parent'] == 38]\n",
    "\n",
    "print('{} subgenres of \"experimental\"'.format(len(experimental)))\n",
    "experimental\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tracks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"10\" halign=\"left\">album</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"10\" halign=\"left\">track</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>comments</th>\n",
       "      <th>date_created</th>\n",
       "      <th>date_released</th>\n",
       "      <th>engineer</th>\n",
       "      <th>favorites</th>\n",
       "      <th>id</th>\n",
       "      <th>information</th>\n",
       "      <th>listens</th>\n",
       "      <th>producer</th>\n",
       "      <th>tags</th>\n",
       "      <th>...</th>\n",
       "      <th>information</th>\n",
       "      <th>interest</th>\n",
       "      <th>language_code</th>\n",
       "      <th>license</th>\n",
       "      <th>listens</th>\n",
       "      <th>lyricist</th>\n",
       "      <th>number</th>\n",
       "      <th>publisher</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>track_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>2008-11-26 01:44:45</td>\n",
       "      <td>2009-01-05 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;p&gt;&lt;/p&gt;</td>\n",
       "      <td>6073</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4656</td>\n",
       "      <td>en</td>\n",
       "      <td>Attribution-NonCommercial-ShareAlike 3.0 Inter...</td>\n",
       "      <td>1293</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>Food</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>2008-11-26 01:44:45</td>\n",
       "      <td>2009-01-05 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;p&gt;&lt;/p&gt;</td>\n",
       "      <td>6073</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1470</td>\n",
       "      <td>en</td>\n",
       "      <td>Attribution-NonCommercial-ShareAlike 3.0 Inter...</td>\n",
       "      <td>514</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>Electric Ave</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0</td>\n",
       "      <td>2008-11-26 01:44:45</td>\n",
       "      <td>2009-01-05 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>&lt;p&gt;&lt;/p&gt;</td>\n",
       "      <td>6073</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1933</td>\n",
       "      <td>en</td>\n",
       "      <td>Attribution-NonCommercial-ShareAlike 3.0 Inter...</td>\n",
       "      <td>1151</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>This World</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0</td>\n",
       "      <td>2008-11-26 01:45:08</td>\n",
       "      <td>2008-02-06 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>NaN</td>\n",
       "      <td>47632</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>54881</td>\n",
       "      <td>en</td>\n",
       "      <td>Attribution-NonCommercial-NoDerivatives (aka M...</td>\n",
       "      <td>50135</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>Freeway</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0</td>\n",
       "      <td>2008-11-26 01:45:05</td>\n",
       "      <td>2009-01-06 00:00:00</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>&lt;p&gt; \"spiritual songs\" from Nicky Cook&lt;/p&gt;</td>\n",
       "      <td>2710</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>978</td>\n",
       "      <td>en</td>\n",
       "      <td>Attribution-NonCommercial-NoDerivatives (aka M...</td>\n",
       "      <td>361</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>Spiritual Level</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 52 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            album                                                     \\\n",
       "         comments         date_created        date_released engineer   \n",
       "track_id                                                               \n",
       "2               0  2008-11-26 01:44:45  2009-01-05 00:00:00      NaN   \n",
       "3               0  2008-11-26 01:44:45  2009-01-05 00:00:00      NaN   \n",
       "5               0  2008-11-26 01:44:45  2009-01-05 00:00:00      NaN   \n",
       "10              0  2008-11-26 01:45:08  2008-02-06 00:00:00      NaN   \n",
       "20              0  2008-11-26 01:45:05  2009-01-06 00:00:00      NaN   \n",
       "\n",
       "                                                                          \\\n",
       "         favorites id                                information listens   \n",
       "track_id                                                                   \n",
       "2                4  1                                    <p></p>    6073   \n",
       "3                4  1                                    <p></p>    6073   \n",
       "5                4  1                                    <p></p>    6073   \n",
       "10               4  6                                        NaN   47632   \n",
       "20               2  4  <p> \"spiritual songs\" from Nicky Cook</p>    2710   \n",
       "\n",
       "                        ...       track                         \\\n",
       "         producer tags  ... information interest language_code   \n",
       "track_id                ...                                      \n",
       "2             NaN   []  ...         NaN     4656            en   \n",
       "3             NaN   []  ...         NaN     1470            en   \n",
       "5             NaN   []  ...         NaN     1933            en   \n",
       "10            NaN   []  ...         NaN    54881            en   \n",
       "20            NaN   []  ...         NaN      978            en   \n",
       "\n",
       "                                                                              \\\n",
       "                                                    license listens lyricist   \n",
       "track_id                                                                       \n",
       "2         Attribution-NonCommercial-ShareAlike 3.0 Inter...    1293      NaN   \n",
       "3         Attribution-NonCommercial-ShareAlike 3.0 Inter...     514      NaN   \n",
       "5         Attribution-NonCommercial-ShareAlike 3.0 Inter...    1151      NaN   \n",
       "10        Attribution-NonCommercial-NoDerivatives (aka M...   50135      NaN   \n",
       "20        Attribution-NonCommercial-NoDerivatives (aka M...     361      NaN   \n",
       "\n",
       "                                                 \n",
       "         number publisher tags            title  \n",
       "track_id                                         \n",
       "2             3       NaN   []             Food  \n",
       "3             4       NaN   []     Electric Ave  \n",
       "5             6       NaN   []       This World  \n",
       "10            1       NaN   []          Freeway  \n",
       "20            3       NaN   []  Spiritual Level  \n",
       "\n",
       "[5 rows x 52 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tracks.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2237 tracks with more than 1000 listens and 5 favorites\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bit_rate</th>\n",
       "      <th>comments</th>\n",
       "      <th>composer</th>\n",
       "      <th>date_created</th>\n",
       "      <th>date_recorded</th>\n",
       "      <th>duration</th>\n",
       "      <th>favorites</th>\n",
       "      <th>genre_top</th>\n",
       "      <th>genres</th>\n",
       "      <th>genres_all</th>\n",
       "      <th>information</th>\n",
       "      <th>interest</th>\n",
       "      <th>language_code</th>\n",
       "      <th>license</th>\n",
       "      <th>listens</th>\n",
       "      <th>lyricist</th>\n",
       "      <th>number</th>\n",
       "      <th>publisher</th>\n",
       "      <th>tags</th>\n",
       "      <th>title</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>track_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>281</th>\n",
       "      <td>256000</td>\n",
       "      <td>0</td>\n",
       "      <td>Brian Chippendale</td>\n",
       "      <td>2008-11-26 01:51:32</td>\n",
       "      <td>2008-11-26 00:00:00</td>\n",
       "      <td>346</td>\n",
       "      <td>11</td>\n",
       "      <td>Experimental</td>\n",
       "      <td>[32]</td>\n",
       "      <td>[32, 38]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2202</td>\n",
       "      <td>en</td>\n",
       "      <td>Attribution-Noncommercial-No Derivative Works ...</td>\n",
       "      <td>2114</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>Juggernaut</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>418</th>\n",
       "      <td>256000</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2008-11-26 01:58:52</td>\n",
       "      <td>2008-10-01 00:00:00</td>\n",
       "      <td>486</td>\n",
       "      <td>10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[1, 18]</td>\n",
       "      <td>[1, 18, 1235, 38]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7458</td>\n",
       "      <td>en</td>\n",
       "      <td>Attribution-NonCommercial-ShareAlike 3.0 Inter...</td>\n",
       "      <td>3890</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>Track1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1066</th>\n",
       "      <td>192000</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2008-11-26 02:33:02</td>\n",
       "      <td>2005-04-15 00:00:00</td>\n",
       "      <td>234</td>\n",
       "      <td>6</td>\n",
       "      <td>Experimental</td>\n",
       "      <td>[22]</td>\n",
       "      <td>[38, 22]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11676</td>\n",
       "      <td>en</td>\n",
       "      <td>Attribution-Noncommercial-Share Alike 3.0 Unit...</td>\n",
       "      <td>10700</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>Helicopter de Cristal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1069</th>\n",
       "      <td>192000</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2008-11-26 02:33:08</td>\n",
       "      <td>2005-04-15 00:00:00</td>\n",
       "      <td>110</td>\n",
       "      <td>6</td>\n",
       "      <td>Experimental</td>\n",
       "      <td>[22]</td>\n",
       "      <td>[38, 22]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7038</td>\n",
       "      <td>en</td>\n",
       "      <td>Attribution-Noncommercial-Share Alike 3.0 Unit...</td>\n",
       "      <td>5661</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>Un Lagrima en la Discoteca</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1076</th>\n",
       "      <td>192000</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2008-11-26 02:33:20</td>\n",
       "      <td>2008-01-01 00:00:00</td>\n",
       "      <td>165</td>\n",
       "      <td>18</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[15, 22]</td>\n",
       "      <td>[38, 22, 15]</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17520</td>\n",
       "      <td>en</td>\n",
       "      <td>Attribution-Noncommercial-Share Alike 3.0 Unit...</td>\n",
       "      <td>14981</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>I Keep Waiting For Earthquakes</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          bit_rate  comments           composer         date_created  \\\n",
       "track_id                                                               \n",
       "281         256000         0  Brian Chippendale  2008-11-26 01:51:32   \n",
       "418         256000         0                NaN  2008-11-26 01:58:52   \n",
       "1066        192000         0                NaN  2008-11-26 02:33:02   \n",
       "1069        192000         0                NaN  2008-11-26 02:33:08   \n",
       "1076        192000         0                NaN  2008-11-26 02:33:20   \n",
       "\n",
       "                date_recorded  duration  favorites     genre_top    genres  \\\n",
       "track_id                                                                     \n",
       "281       2008-11-26 00:00:00       346         11  Experimental      [32]   \n",
       "418       2008-10-01 00:00:00       486         10           NaN   [1, 18]   \n",
       "1066      2005-04-15 00:00:00       234          6  Experimental      [22]   \n",
       "1069      2005-04-15 00:00:00       110          6  Experimental      [22]   \n",
       "1076      2008-01-01 00:00:00       165         18           NaN  [15, 22]   \n",
       "\n",
       "                 genres_all information  interest language_code  \\\n",
       "track_id                                                          \n",
       "281                [32, 38]         NaN      2202            en   \n",
       "418       [1, 18, 1235, 38]         NaN      7458            en   \n",
       "1066               [38, 22]         NaN     11676            en   \n",
       "1069               [38, 22]         NaN      7038            en   \n",
       "1076           [38, 22, 15]         NaN     17520            en   \n",
       "\n",
       "                                                    license  listens lyricist  \\\n",
       "track_id                                                                        \n",
       "281       Attribution-Noncommercial-No Derivative Works ...     2114      NaN   \n",
       "418       Attribution-NonCommercial-ShareAlike 3.0 Inter...     3890      NaN   \n",
       "1066      Attribution-Noncommercial-Share Alike 3.0 Unit...    10700      NaN   \n",
       "1069      Attribution-Noncommercial-Share Alike 3.0 Unit...     5661      NaN   \n",
       "1076      Attribution-Noncommercial-Share Alike 3.0 Unit...    14981      NaN   \n",
       "\n",
       "          number publisher tags                           title  \n",
       "track_id                                                         \n",
       "281            4       NaN   []                      Juggernaut  \n",
       "418            1       NaN   []                          Track1  \n",
       "1066           1       NaN   []           Helicopter de Cristal  \n",
       "1069           4       NaN   []      Un Lagrima en la Discoteca  \n",
       "1076          11       NaN   []  I Keep Waiting For Earthquakes  "
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the genre and listens of tracks with genre \"experimental\"\n",
    "bad_genres = [38]\n",
    "experimental_tracks = tracks.loc[tracks['track', 'genres_all'].apply(lambda x: any(val in ast.literal_eval(x) for val in bad_genres))]\n",
    "\n",
    "# filter to the tracks with more than 1000 listens and 5 favorites\n",
    "experimental_tracks = experimental_tracks.loc[(experimental_tracks['track', 'listens'] > 1000) & (experimental_tracks['track', 'favorites'] > 5)]\n",
    "\n",
    "print('{} experimental tracks with more than 1000 listens and 5 favorites'.format(len(experimental_tracks)))\n",
    "experimental_tracks['track'].head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "getting indices for: 35917 tracks\n",
      "[137, 138, 148, 149, 150] ...\n"
     ]
    }
   ],
   "source": [
    "# Now, get the indices of the worst experimental tracks. We can pass these into the ignore argument of our preprocessor\n",
    "\n",
    "experimental_genre = 38\n",
    "experimental_tracks = tracks.loc[tracks['track', 'genres_all'].apply(lambda x: experimental_genre in ast.literal_eval(x))]\n",
    "experimental_tracks = list(experimental_tracks.loc[(experimental_tracks['track', 'listens'] <= 1000) | (experimental_tracks['track', 'favorites'] <= 5)].index)\n",
    "print(f'getting indices for: {len(experimental_tracks)} tracks')\n",
    "print(f'{experimental_tracks[:5]} ...')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.034907853877003264"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# new dataset makeup:\n",
    "2237 / (100000 - 35917)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After this removal process there will be about 2.2k experimental tracks left, which is 3% of the new dataset (~65k clips)\n",
    "\n",
    "Unfiltered, there are 36k, which is about 36% of the dataset."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "open-musiclm",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
